{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    return 3*x**2 - 4*x + 5"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "xs = np.arange(-5, 5, 0.25)\n",
    "ys = f(xs)\n",
    "plt.plot(xs, ys)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "h = 0.000001\n",
    "x = 2/3\n",
    "(f(x + h) - f(x))/h"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# les get more complex\n",
    "\n",
    "h = 0.0001\n",
    "\n",
    "# inputs\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = a*b + c\n",
    "c += h\n",
    "d2 = a*b + c\n",
    "\n",
    "print('d1', d1)\n",
    "print('d2', d2)\n",
    "print('slope', (d2 - d1)/h)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Value:\n",
    "\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0\n",
    "        self._backward = lambda: None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data})\"\n",
    "\n",
    "    def __add__(self, other):\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += 1.0 * out.grad\n",
    "            other.grad += 1.0 * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad\n",
    "            other.grad += self.data * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self, ), 'tanh')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "\n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "a = Value(2.0, label='a')\n",
    "b = Value(-3.0, label='b')\n",
    "c = Value(10.0, label='c')\n",
    "e = a*b; e.label = 'e'\n",
    "d = e + c; d.label = 'd'\n",
    "f = Value(-2.0, label='f')\n",
    "L = d * f; L.label = 'L'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "Value(data=-8.0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "    # builds a set of all nodes and edges in a graph\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for child in v._prev:\n",
    "                edges.add((child, v))\n",
    "                build(child)\n",
    "    build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "def draw_dot(root):\n",
    "    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
    "\n",
    "    nodes, edges = trace(root)\n",
    "    for n in nodes:\n",
    "        uid = str(id(n))\n",
    "        # for any value in the graph, create a rectangular ('record') node for it\n",
    "        dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "        if n._op:\n",
    "            # if this value is a result of some operation, create an op node for it\n",
    "            dot.node(name = uid + n._op, label = n._op)\n",
    "            # and connect this node to it\n",
    "            dot.edge(uid + n._op, uid)\n",
    "\n",
    "    for n1, n2 in edges:\n",
    "        # connect n1 to the op node of n2\n",
    "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "\n",
    "    return dot"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "draw_dot(L)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "a.data += 0.01 * a.grad\n",
    "b.data += 0.01 * b.grad\n",
    "c.data += 0.01 * c.grad\n",
    "f.data += 0.01 * f.grad\n",
    "\n",
    "e = a * b\n",
    "d = e + c\n",
    "L = d * f\n",
    "\n",
    "print(L.data)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def lol():\n",
    "\n",
    "    h = 0.001\n",
    "\n",
    "    a = Value(2.0, label='a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    c = Value(10.0, label='c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L1 = L.data\n",
    "\n",
    "    a = Value(2.0, label='a')\n",
    "    b = Value(-3.0, label='b')\n",
    "    b.data += h\n",
    "    c = Value(10.0, label='c')\n",
    "    e = a*b; e.label = 'e'\n",
    "    d = e + c; d.label = 'd'\n",
    "    f = Value(-2.0, label='f')\n",
    "    L = d * f; L.label = 'L'\n",
    "    L2 = L.data\n",
    "\n",
    "    print((L2 - L1)/h)\n",
    "\n",
    "lol()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plt.plot(np.arange(-5,5,0.2), np.tanh(np.arange(-5,5,0.2))); plt.grid();"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# inputs x1,x2\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "# weights w1,w2\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "# bias of the neuron\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "# x1*w1 + x2*w2 + b\n",
    "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label = 'n'\n",
    "o = n.tanh(); o.label = 'o'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "draw_dot(o)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "o.backward()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "topo = []\n",
    "visited = set()\n",
    "def build_topo(v):\n",
    "    if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "            build_topo(child)\n",
    "        topo.append(v)\n",
    "build_topo(o)\n",
    "# topo\n",
    "\n",
    "# [Value(data=6.881373587019543),\n",
    "#  Value(data=2.0),\n",
    "#  Value(data=-3.0),\n",
    "#  Value(data=-6.0),\n",
    "#  Value(data=0.0),\n",
    "#  Value(data=1.0),\n",
    "#  Value(data=0.0),\n",
    "#  Value(data=-6.0),\n",
    "#  Value(data=0.8813735870195432),\n",
    "#  Value(data=0.7071067811865476)]\n",
    "\n",
    "o.grad = 1.0"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "o._backward()\n",
    "\n",
    "n._backward()\n",
    "\n",
    "b._backward()\n",
    "\n",
    "x1w1x2w2._backward()\n",
    "\n",
    "x2w2._backward()\n",
    "x1w1._backward()\n",
    "\n",
    "x1.grad = w1.data * x1w1.grad\n",
    "w1.grad = x1.data * x1w1.grad\n",
    "\n",
    "x2.grad = w2.data * x2w2.grad\n",
    "w2.grad = x2.data * x2w2.grad\n",
    "\n",
    "x1w1.grad = 0.5\n",
    "x2w2.grad = 0.5\n",
    "\n",
    "x1w1x2w2.grad = 0.5\n",
    "b.grad = 0.5\n",
    "\n",
    "n.grad = 0.5\n",
    "o.grad = 1.0\n",
    "\n",
    "1 - o.data**2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n",
    "# o = tanh(n)\n",
    "# do/dn = 1 - o**2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "a = Value(3.0, label='a')\n",
    "b = a + a   ; b.label = 'b'\n",
    "b.backward()\n",
    "draw_dot(b)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "a = Value(-2.0, label='a')\n",
    "b = Value(3.0, label='b')\n",
    "d = a * b    ; d.label = 'd'\n",
    "e = a + b    ; e.label = 'e'\n",
    "f = d * e    ; f.label = 'f'\n",
    "\n",
    "f.backward()\n",
    "\n",
    "draw_dot(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}